package wordstat;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URLEncoder;
import java.util.*;

public class WordstatParse {
    //    public static final String ENC = "windows-1251";
    public static final String ENC = "UTF-8";
    public static final int TIMEOUT = 20000;
    public static final int DELAY_REQUEST = 12000;
    public static final int DELAY_PHRASE = 6000;

    private static final int MAX_COUNT = 10;
    private static final String GEO_CODE = "10174";
    private static final String GEO_TEXT = "Санкт-Петербург+и+Ленинградская+область";
    public static final String BASE_URL = "http://wordstat.yandex.ru/?cmd=words";

//    private static final int MAX_COUNT = 100;
//    private static final String GEO_CODE = "";
//    private static final String GEO_TEXT = "";

    public static void main(String[] args) throws IOException, InterruptedException {

        BufferedReader reader = new BufferedReader(new FileReader("wordstat.phrases.txt"));
        String[] initPhrases = reader.readLine().split(",");
        int count = 0;
        List<String> allUniqueWords = new ArrayList<String>();
        PrintStream writerUnique = new PrintStream(new FileOutputStream("wordstat/all-unique.txt"));
        StringBuffer sbUnique = new StringBuffer();
        String geoCode = GEO_CODE;
        for (String phrase : initPhrases) {
            if (count++ != 0) {
                Thread.sleep(DELAY_REQUEST);
            }
            PrintStream writerPositions = new PrintStream(new FileOutputStream("wordstat\\" + phrase.trim() + ".txt"));
            PrintStream writerPhrases = new PrintStream(new FileOutputStream("wordstat\\" + phrase.trim() + "-phrases.txt"));
            int page = 1;
            List<String> title = new ArrayList<String>();
            label:
            while (true) {
//                Elements phrases = requestPhrases(geoCode, phrase, GEO_TEXT, page);
                Map<String, Integer> phrases = requestPhrases(geoCode, phrase, GEO_TEXT, page);
                if (phrases == null) {
                    break;
                }
                for (String p : phrases.keySet()) {
                    String[] fields = p.split(" ");
                    int requestCount = phrases.get(p);
//                }
//                for (Element p : phrases) {
//                    String[] fields = p.text().split(" ");
//                    int requestCount = Integer.parseInt(fields[fields.length - 1]);
//                    if (requestCount < MAX_COUNT) {
//                        break label;
//                    }
                    StringBuffer sb = new StringBuffer();
                    for (int i = 0; i < fields.length - 1; i++) {
                        String field = fields[i].replace("+", "");
                        if (!containsStop(field) && !title.contains(field)) {
                            title.add(field);
                        }
                        if (!containsStop(field) && !allUniqueWords.contains(field)) {
                            allUniqueWords.add(field);
                            sbUnique.append(" ").append(field);
                        }
                        sb.append(" ").append(field);
                    }
                    sb.append(",").append(requestCount);
                    sb.replace(0, 1, "");
                    if (!containsStop(sb.toString())) {
                        System.out.println(sb);
                        writerPositions.println(sb);
                        fields = sb.toString().split(",");
                        writerPhrases.println(fields[0]);
                    } else {
                        System.out.println("STOP: " + sb);
                    }
                }
                Thread.sleep(DELAY_PHRASE);
                page++;
            }
            sbUnique.append("\n");
            StringBuffer titleBuff = new StringBuffer();
            for (String titleElem : title) {
                titleBuff.append(" ").append(titleElem);
            }
            titleBuff.replace(0, 1, "");
            writerPositions.println(titleBuff);
            System.out.println(titleBuff);
            System.out.println("-----------------------------------------------------------");
            writerPositions.close();
            writerPhrases.close();
        }
        writerUnique.println(sbUnique);
        System.out.println(sbUnique);
        writerUnique.close();
    }

    public static int requestTopRequestCount(String geoCode, String phrase, String textGeo) throws IOException, InterruptedException {
        try {
            Map<String, Integer> phrases = requestPhrases(geoCode, phrase, textGeo, 1);
            if (phrases == null) {
                return 0;
            }
            return phrases.entrySet().iterator().next().getValue();
        } finally {
            Thread.sleep(DELAY_PHRASE);
        }
    }

    private static  Map<String, Integer> requestPhrases(String geoCode, String phrase, String textGeo, int page) throws IOException, InterruptedException {
        String text = URLEncoder.encode(phrase, ENC);
        textGeo = URLEncoder.encode(textGeo, ENC);
        Document doc;
        while (true) {
            try {
                String readyURL = BASE_URL + "&page=" + page + "&t=" + text + "&geo=" + geoCode + "&text_geo=" + textGeo;
                doc = Jsoup.connect(readyURL).
                        header("user-agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0").
                        header("Cookie", "yp=1628953306.sp.nd:50:ws:1:lang:!#1628139604.udn.c2hlbXBsaW5lcg==; yandexuid=36261312612900; fuid01=4d726ae12b1a1ac0.N9hJY3OOtWG-kEtar5NvfvMUs1Xo_8AUqbE-fG1uUYhwf2PZq1e_rMNyTIPcAliV5rOj5oRQaNMCW7Z2qn2HXaeA_Otjy40U2yeL3hFKkrbBAo6y88nb2eM2jOvZk-KQ; yabs-frequency=/3/hNO30ni9Pm000tUn01qR2Jm0005t0W0N60dS8t033HW9XPMj0mqO2Pm0/; my=YwECAAAnAgAALQECAA==; L=A39SC35HeVJlX05NeVpTUwMBAFh/bGdcNhoKMCoZYjE8UyAONRw9GgkaWCIdAnAeARQAOz8SMRpjBHdCJjNgWw==.1312779604.9074.268430.03ebff42cc506f579abb06a175673d47; spravka=dD0xMzEyNjQ5MjM4O2k9MTA5LjIwNS4yNTMuMTM1O3U9MTMxMjY0OTIzODkzNzY5MTU3MTtoPTY0M2FlZGZjYzkyZWQ2Mjk3OTIxOTUyYzhiYjk1Mjk2; t=p; yandex_login=shempliner; yandex_mail=shempliner; ys=udn.c2hlbXBsaW5lcg==; Session_id=1312779604.645.0.118550753.2:330838428:230.8:1312779604611:1388338972:13.77466.36850.267a73e88a26321cca687cc4f888684e").
                        timeout(TIMEOUT).
                        get();
                break;
            } catch (IOException e) {
                Thread.sleep(DELAY_PHRASE);
                System.out.println(e.getMessage() + " Reconnect..");
            }
        }
        Elements phraseElems = doc.select("td:lt(1) tr.tlist");
        if (phraseElems.size() <= 0) {
            return null;
        }
        Map<String, Integer> phrasesMap = new LinkedHashMap<String, Integer>(phraseElems.size());
        for (Element p : phraseElems) {
            String[] fields = p.text().split(" ");
            int requestCount = Integer.parseInt(fields[fields.length - 1]);
            StringBuilder sb = new StringBuilder();
            for (int i = 0; i < fields.length - 1; i++) {
                sb.append(" ").append(fields[i]);
            }
            sb.delete(0, 1);
            phrasesMap.put(sb.toString(), requestCount);
        }
        return phrasesMap;
    }

    public static boolean containsStop(String phrase) {
        for (String stopPhrase : stopPhrases) {
            if (phrase.contains(stopPhrase)) {
                return true;
            }
        }
        return false;
    }

    public static final String[] stopPhrases = new String[]{
            "фото",
            "видео",
            "скач",
            "книг",
            "руками",
            "своими",
            "руководство",
            "самостоятельн",
            "пушкин",
            "колпин",
//            "программ",
            "игр",
            "курсовая",
            "бесплатн",
            "как",
            "где",
            "время",
            "выставк",
            "вакансии",
            "колледж",
            "курсы",
            "вуз",
            "факультет",
            "онлайн",
//            "компьютерн",
            "ищу",
//            "школа",
//            "обучени",
            "отзывы",
            "инструкци",
            "детск",
            "сочи",
            "рославль",
            "форум",
            "москв",
            "петрозаводск",
            "архангельск",
            "екатеринбург",
            "оренбург",
            "тольятти",
            "вологд",
            "твер",
            "новосиб",
            "схема",
            "смоленск",
            "калининград",
            "костром",
            "чебоксар",
            "череповец",
            "иваново",
            "юнона",
            "оквэд",
            "пособие",
            "самоучитель",
            "закон",
            "гарантийн",
            "гаранград",
//            "сервисн",
            "мурманск",
            "нижний",
            "новгород",
            "разряжается",
            "садится",
            "проверка",
            "внешний",
            "проверить",
            "зарядить",
            "вздулся",
            "проблема",
            "распиновка",
            "велосипед",
            "гнезд",
            "полярность",
            "сломал",
            "сколько",
            "распайка",
            "индикатор",
            "насос",
            "усилитель",
            "сейф",
            "bravo",
            "topaz",
            "ballu",
            "picocell",
            "superturbo",
            "hpp",
            "bratsk",
            "calcell",
            "adagio",
            "софт"
    };
}